
import requests,re,time,json

# 存储爬取信息的列表
t58_info = list()
# 58同城用G#x编码的一些奇异字符表示数字,网站可能会变动
t58_dict = {'龒':'0','龤':'1','驋':'2','閏':'3','餼':'4','麣':'5','鑶':'6','龥':'7','齤':'8','鸺':'9'}

def get_session():
    """获取一个设置了头部的session,自动保持cookie"""
    s = requests.Session()
    s.headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'accept-language': 'zh-CN,zh;q=0.9',
        'referer': 'https://sz.58.com/chuzu/?PGTID=0d100000-008c-78a5-ac77-a44cc3330ee8&ClickID=2',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    }
    s.get(url='https://zh.58.com/chuzu/')
    return s


def convert(s):
    """把&#x转换为ｕtf8最后转换为数字"""
    s = s.strip('&#x;')  # 例如把'&#x957f;'变成'957f'
    s = bytes(r'\u' + s, 'ascii')  # 把'xxxx'转换成b'\uxxxx'
    s = s.decode('unicode_escape')  # 把bytes转换为字符串,为奇异汉字
    s = t58_dict.get(s, '')  # 将奇异汉字转换为数字
    return s


def get_content(html, info=t58_info):
    """解析具体的网页，存入列表中"""
    # 替换换行和&#x
    html = html.replace('\n', '')
    html = re.sub(r'&#x[a-f0-9]{4};', lambda match: convert(match.group()), html)
    # 获取li块列表
    ul = re.search(r'<ul class="listUl">(.*?)</ul>', html).group(1)
    lis = re.findall(r'<li logr=".*?"\s+sortid=".*?">(.*?)</li>', ul)
    # 遍历li块列表,取具体的内容,存入列表中, 解析出错会打印错误并继续执行
    for li in lis:
        try:
            temp = dict()
            temp['img'] = "https:" + re.search(r'<img\s+lazy_src="(.*?)"\s+src=".*?">', li).group(1)
            temp['name'] = re.search(r'<a href=".*?"\s+class="strongbox"\s+tongji_label="listclick".*?>\s*(.*?)\s*</a>', li).group(1)
            temp['money'] = re.search(r'<div class="money">\s*?<b class="strongbox">(.*?)</b>元/月\s*?</div>', li).group(1)
            ret = re.search(r'<p class="room strongbox">(.*?)\s+(&nbsp;)+(.*?)</p>', li)
            temp['house'] = ret.group(1) + ret.group(3)
            info.append(temp)
        except Exception as err:
            print(err)
    # 写入文件中，根据列表长度一条条分行页
    with open('58result_houser.txt', 'w', encoding='utf-8') as f:
        for i in range(len(info)):
            f.write(json.dumps(info[i], ensure_ascii=False)+"\n")

def get_link(session, pindex=1):
    """爬取深圳租房信息,pindex为页码"""
    url = 'https://zh.58.com/chuzu/pn{}/?PGTID=0d3090a7-008c-73bb-6037-bfeefa80b85e&ClickID=2'.format(int(pindex))
    try:
        res = session.get(url=url)
        if res.status_code != 200:
            raise Exception('状态码非200')
        get_content(res.text)
    except Exception as err:
        print(err)


def main():
    try:
        pindex = int(input("请输入爬取的页码(无效输入均默认为１):"))
    except:
        pindex = 1 # 输入有错则默认为1
    s = get_session()
    get_link(session=s, pindex=pindex)

if __name__ == '__main__':
    main()